#! /usr/bin/python
#####################################################################################################
# Target   : Autoconvert J750 tester ASCII datalog file to customer data                            #
# Author   : Roger Zheng                                                                            #
# Version  : 00                                                                                     #
# Date     : 01/Jul/06                                                                              #
# Location : BeiJing,China                                                                          #
#---------------------------------------------------------------------------------------------------#
#Instruction:                                                                                       #
#---------------------------------------------------------------------------------------------------#
#Filename Format: Devicename_Version_Sitnum_TestType_LotWaferId_Date_Time_Testername.data           #
#Configure File Format:                                                                             #
# Device:Version:Patch:Sortnum:Total_test_itemnum:Parameter_fuction_itemnum:bindefine:testitem_name #
#Directory Sturcture:                                                                               #
#  /TestData                                                                                        #
#       |---/Device1                                                                                #
#       |       |-----/Lot1                                                                         #
#       |       |       |---/Data   # Put original test data in this directory                      # 
#       |       |       |     |------data file1                                                     #
#       |       |       |     |------data file2                                                     #
#       |       |       |     .................                                                     #
#       |       |       |---/Retest #Put Retest data in this directory                              #
#       |       |       |     |------retest data file1                                              #
#       |       |       |     .................                                                     #
#       |       |       |---/bhtc                                                                   #
#       |       |       |---/map                                                                    #
#       |       |       |---/maphh                                                                  #
#       |       |-----/Lot2                                                                         #
#       |       ...............................                                                     #
#       |---/Device2                                                                                #
#       .......................................                                                     #
#                                                                                                   #
#Initialize Direcotry & File :                                                                      #
#$dir_testdata,$dir_confige,$file_passlotlog,$file_passwaferlog,$file_errorlog                      #
#Output:BHTC,MAP,MAPHH                                                                              #
#***************************************************************************************************#
#The script can autocovert the all target directory data to customer data. and skip the wafer or    #
#lot and that have coverted. If you want to reconvert some wafer data, you must delete the wafer    #
#information from passwafer log file and passlot log file according to data file!                   #
#####################################################################################################

####################################################################
#Note:Sort0 File in Retest directory is as Sort2 retest file
#

##############################################################################
#rev02:   2006/12/01
#         fixed bug that don't read first testdata when bhtc is exist
#         Auto Unzip Retest data file if Retest data files have been zipped

import os
import sys
import time
import glob
import re
import string
import shutil
import zipfile

from Configure_ver00 import Config
from Data2Bhtc_ver01 import NewUiData2BHTC
from BHTC2CustData_ver00  import BHTC2CustData
#from renamefile import Rename

def MyZipFile(ZipFileName,FileName,FlagDelFile):
	if not os.path.isfile(ZipFileName):
		FileNameFH=zipfile.ZipFile(ZipFileName,"w")
		FileNameFH.write(FileName,None,zipfile.ZIP_DEFLATED)
		FileNameFH.close()
		
	else:
		FlagFileExist=False
		FileNameFH=zipfile.ZipFile(ZipFileName,"r")
		for TempFileName in FileNameFH.namelist():
			if re.search(FileName,TempFileName,re.I):
				FlagFileExist=True
				break
		FileNameFH.close()
		if not FlagFileExist:    
			FileNameFH=zipfile.ZipFile(ZipFileName,"a")
			FileNameFH.write(FileName,None,zipfile.ZIP_DEFLATED)
			FileNameFH.close()
	if FlagDelFile:
		os.unlink(FileName)
		
def main():
	#######################################
	#Edit Area
	######################################
	ConfigureFile="Config_NewUi"
	#CurrentPath=os.getcwd()
	CurrentDir=os.path.dirname(CurrentPath)
	TestDataPath=os.path.join(CurrentDir,"NewTestData")
	ConfigPath=os.path.join(CurrentPath,"Config")
	LogPath=os.path.join(CurrentPath,"Log")
	BinDefinePath=os.path.join(CurrentPath,"Parameter")
	TempPath=os.path.join(CurrentPath,"Tmp")
	
	###########################
	#Find InitSetting_Auto for TestDataDir
	###########################
	InitSettingFile=os.path.join(ConfigPath,"InitSetting_Auto")
	os.chdir(ConfigPath)
	if os.path.isfile(InitSettingFile):
		InitSettingFileFH=open(InitSettingFile)
		for Line in InitSettingFileFH.readlines():
			if re.search(r"TestDataDir=",Line,re.I):
				Line=Line.strip()  
				TempArray=re.split(r"=",Line)
				TestDataPath=TempArray[1]
				break
	if not os.path.isdir(TempPath):
		os.mkdir(TempPath)
	if not os.path.isdir(os.path.join(TempPath,"orgbhtc")):
		os.mkdir(os.path.join(TempPath,"orgbhtc"))
	
	##############################################
	#Initial file directory!
	##############################################  
	TempFileBHTC=os.path.join(TempPath,"bhtc_auto")
	TempOrgFileBHTC=os.path.join(TempPath,"orgbhtc")
	ErrorLogFile=os.path.join(LogPath,"error.log")
	
	try:
		ErrorFH=open(ErrorLogFile,"a")
		
	except IOError:
		print "Error Log File can't find in ",LogPath
		#ErrorFH.close()
		sys.exit()
		
	TimeConvert=time.asctime()
	
	try:
		os.chdir(TestDataPath)
	
	except OSError:
		ErrorFH=open(ErrorLogFile,"a")
		ErrorFH.write("%20s  %s%s%s" % (str(TimeConvert),"The TestDataPath:\"",TestDataPath,"\"isn't exist!\n")),
		ErrorFH.close()
		sys.exit()
		
		
	#############################################################
	#covert all data to customer data of every directory
	############################################################  
	ExisitDevices=glob.glob("*")
	
	for DeviceName in ExisitDevices:
		DevicePath=os.path.join(TestDataPath,DeviceName)
		if os.path.isdir(DevicePath):    
			os.chdir(DevicePath)    
			ExisitLots=glob.glob("*")
			FlagPassLot=False 
			#PassLotLogFH.write("hhh\n")                    
			
			for Lot in ExisitLots: 
				##################################################
				#Juadge if the lot have coverted
				##################################################
				try:
					PassLotLogFH=open(os.path.join(LogPath,"passlot.log"),"r")
				except IOError:
					ErrorFH=open(ErrorLogFile)
					ErrorFH.write("%s%s%s"% ("Can't find logfile: passlot.log in directory:",LogPath,"\n"))
					ErrorFH.close()
				for LotLogLine in PassLotLogFH.readlines():
					if re.search(Lot,LotLogLine,re.I) and re.search(DeviceName,LotLogLine,re.I):
						FlagPassLot=True
						break
				PassLotLogFH.close      
				##################################################
				#Start new lot that don't converted!
				################################################## 
				LotPath=os.path.join(DevicePath,Lot)
				if (not FlagPassLot) and os.path.isdir(LotPath): 
					DataPath=os.path.join(LotPath,"data") 
					RetestDataPath=os.path.join(LotPath,"Retest")
					os.chdir(DataPath)
					###############################################
					#For Old UI because the file format is txt
					###############################################
					#if len(glob.glob("*.txt")):
					#  Rename(DataPath,"txt","data")
			
					ExisitWaferFiles=sorted(glob.glob("*.data"))            
					for WaferFile in ExisitWaferFiles:
						TempDataFileName=re.split(r"_",WaferFile)
						DeviceName=TempDataFileName[0]          
						SortType=TempDataFileName[3]
						LotWaferId=TempDataFileName[4]
						Version=""
						Patch=""
						TesterName=""
						OperatorName=""
						ChannelMap=""
						Title=""
						StarTime=""
						TestItemNum=""
						DigitalTestItemNum=""
						TestItemName=""
						FileBHTC=""
						RetestDataFiles=[]
						FlagRetest=False
						FlagRetestDataFileExist=False
						FlagExisitBHTC=False
						FlagTest="data"
						FlagConfig=False
						#FlagData=False
						FlagRun=0
						FlagPassWafer=False
						TimeConvert=time.asctime()
						#print TimeConvert
						###################################################
						#Juadge if the wafer have coverted
						###################################################
						TempLotWaferId=re.split("-",LotWaferId)
						LotNum=TempLotWaferId[0]
						WaferId=int(TempLotWaferId[1])
						if len(TempLotWaferId[1])<2:
							LotWaferId="".join((TempLotWaferId[0],"-","0",TempLotWaferId[1]))
							
						try:
							PassWaferLogFH=open(os.path.join(LogPath,"passwafer.log"),"r")
						except IOError:
							print "Cant't find open logfile: passwafer.log in directory:",LogPath,"\"\n"
						#PassWaferLogFH=open(u"e:\\GUI\\dataprocess\\PythonScript\\log\\passwafer.log","rw")  
						#PassWaferLogLines=PassWaferLogFH.readlines()
						for PassWaferLogLine in PassWaferLogFH.readlines():
							if re.search("^#",PassWaferLogLine,re.I) or len(re.findall("\|",PassWaferLogLine))<3:
								continue            
							TempArrayDatas=re.split("\|",PassWaferLogLine)
							Temp=re.split("-",TempArrayDatas[3])            
							if (LotNum == Temp[0].strip())and(WaferId==int(Temp[1])) and (SortType == TempArrayDatas[4].strip())and (DeviceName == TempArrayDatas[1].strip()):
								FlagPassWafer=True
								break
						PassWaferLogFH.close  
						##################################################
						#Start new wafer that don't converted!
						################################################## 
						if not FlagPassWafer:  
							######################################################
							#Got all configure parameter from configure file
							######################################################
							(FlagConfig,Version,Patch,TestItemNum,DigitalTestItemNum,TestItemNames)=Config(DeviceName,SortType,ConfigPath,ConfigureFile)            
							if FlagConfig:                          
								##################################
								#Delete Temp bhtc File if exists
								##################################
								if os.path.isfile(TempFileBHTC):
									os.unlink(TempFileBHTC)
								if os.path.isdir(RetestDataPath):
									os.chdir(RetestDataPath)
									ZipFileName="".join((DeviceName,"_",Version,"_",os.path.basename(LotPath),"_",SortType,"_","Retest",".zip"))
									if os.path.isfile(ZipFileName):
										FileName=zipfile.ZipFile(ZipFileName,"r")
										for TempFile in FileName.namelist():
											TempArrayDatas=re.split("_",TempFile)
											Temp=re.split("-",TempArrayDatas[4])
											if string.upper(SortType)==string.upper("Sort1"):
												if (re.search(LotNum,TempFile,re.I)) and (WaferId ==int(Temp[1])) and (string.upper(SortType)==string.upper(TempArrayDatas[3])):
													RetestDataFile=TempFile
													RetestDataFileFH=open(RetestDataFile,"w")
													RetestDataFileFH.write(FileName.read(RetestDataFile))
													RetestDataFileFH.close()
											else:
												if(re.search(LotNum,TempFile,re.I)) and (WaferId==int(Temp[1]))and (string.upper("Sort0")==string.upper(TempArrayDatas[3])or String.upper(SortType)==string.upper(TempArrayDatas[3])):
													RetestDataFile=TempFile
													RetestDataFileFH=open(RetestDataFile,"w")
													RetestDataFileFH.write(FileName.read(RetestDataFile))
													RetestDataFileFH.close()
												   
										FileName.close()	
									###############################################
									#For Old UI because the file format is txt
									###############################################
									#if len(glob.glob("*.txt")):
									#  Rename(RetestDataPath,"txt","data")
									TempRetestDataFiles=glob.glob("*.data")
									TempFiles=glob.glob("*.data")
									Date=[]
									Time=[]
									RetestDataFileNum=0
									###################################################
									#Find the all retest file of the related wafer
									#and sort the retest files by tested time
									###################################################
									
									for TempRetestDataFile in TempFiles:
										TempArrayDatas=re.split("_",TempRetestDataFile)
										Temp=re.split("-",TempArrayDatas[4])
										ReSortType="Sort0"                  
										if (re.search(LotNum,TempRetestDataFile,re.I))and (WaferId ==int(Temp[1]))and (string.upper(SortType) == string.upper(TempArrayDatas[3]) or string.upper(ReSortType)==string.upper(TempArrayDatas[3])) :
											TempDatas=re.split(r"_|\.",TempRetestDataFile)
											Date.append(TempDatas[5])
											Time.append(TempDatas[6])
											RetestDataFileNum=RetestDataFileNum+1
											FlagRetest=True
										else:
											TempRetestDataFiles.remove(TempRetestDataFile)
									TempDate=""
									TempTime=""
									
									for Index1 in range(RetestDataFileNum):
										for Index2 in range(RetestDataFileNum):                            
											if int(Date[Index1])<int(Date[Index2]):
												TempDate=Date[Index2]
												TempTime=Time[Index2]
												Date[Index2]=Date[Index1]
												Time[Index2]=Time[Index1]
												Date[Index1]=TempDate
												Time[Index1]=TempTime
											elif int(Date[Index1])<int(Date[Index2]):  
												if int(Time[Index1])<int(Time[Index2]):
													TempDate=Date[Index2]
													TempTime=Time[Index2]
													Date[Index2]=Date[Index1]
													Time[Index2]=Time[Index1]
													Date[Index1]=TempDate
													Time[Index1]=TempTime
									#RetestDataFiles=[] 
									for Index in range(RetestDataFileNum):                
										for TempFile in TempRetestDataFiles:                  
											if re.search(Date[Index],TempFile,re.I) and re.search(Time[Index],TempFile,re.I):
												RetestDataFiles.append(TempFile)
												TempRetestDataFiles.remove(TempFile)
												
								######################################################
								#Find if there is BHTC file when retest data exist
								######################################################
								BHTCPath=os.path.join(LotPath,"bhtc")
									
								if os.path.isdir(BHTCPath):
									os.chdir(BHTCPath)
									TempBHTCFiles=glob.glob("*.bhtc")
									for TempFile in TempBHTCFiles:
										TempArrayDatas=re.split("_|\.",TempFile)                  
										if re.search(LotNum,TempFile,re.I)and (WaferId==int(TempArrayDatas[-2]))and (SortType==TempArrayDatas[-3]):
											shutil.copyfile(TempFile,TempFileBHTC)                      
											FileBHTC=os.path.join(BHTCPath,TempFile)
											FlagExisitBHTC=True
											break
								#########################################
								#Covert data
								#########################################
								if FlagExisitBHTC and FlagRetest:
									shutil.move(FileBHTC,TempOrgFileBHTC)   
								(FlagRun,StarTime,ProgramName,TesterName,Operator,ChannelMap)=NewUiData2BHTC(LotPath,WaferFile,TempFileBHTC,TestItemNum,TestItemNames,DigitalTestItemNum,FlagTest)  
								###############################################
								#Zip the converted data file 
								###############################################
								if not FlagRun:
									os.chdir(os.path.join(LotPath,FlagTest))
									if re.search("Sort1",SortType,re.I):
										ZipFileName="".join((DeviceName,"_",Version,"_",os.path.basename(LotPath),"_",SortType,"_",FlagTest,".zip"))
										MyZipFile(ZipFileName,WaferFile,True)                      
									elif re.search("Sort2",SortType,re.I) or re.search("Sort0",SortType,re.I):
										ZipFileName="".join((DeviceName,"_",Version,"_",os.path.basename(LotPath),"_","Sort2","_",FlagTest,".zip"))
										MyZipFile(ZipFileName,WaferFile,True)
								if FlagRetest:
									TempFlagRun=0
									TempFlag=0
									FlagTest="Retest"
									for RetestDataFile in RetestDataFiles:
										TempDataFileName=re.split(r"_",RetestDataFile)
										DeviceName=TempDataFileName[0]          
										SortType=TempDataFileName[3]
										#LotWaferId=TempDataFileName[4]
										(FlagConfig,Version,Patch,TestItemNum,DigitalTestItemNum,TestItemNames)=Config(DeviceName,SortType,ConfigPath,ConfigureFile)                  
										if string.upper(SortType)==string.upper("Sort0"):
											SortType="Sort2"
										(TempFlag,ProgramName,TesterName,Operator,ChannelMap)=NewUiData2BHTC(LotPath,RetestDataFile,TempFileBHTC,TestItemNum,TestItemNames,DigitalTestItemNum,FlagTest)  
										###############################################
										#Zip the converted retest data file 
										###############################################
										os.chdir(os.path.join(LotPath,FlagTest))
										if re.search("Sort1",SortType,re.I):
											ZipFileName="".join((DeviceName,"_",Version,"_",os.path.basename(LotPath),"_",SortType,"_",FlagTest,".zip"))
											MyZipFile(ZipFileName,RetestDataFile,True)
										elif re.search("Sort2",SortType,re.I) or re.search("Sort0",SortType,re.I):
											ZipFileName="".join((DeviceName,"_",Version,"_",os.path.basename(LotPath),"_","Sort2","_",FlagTest,".zip"))
											MyZipFile(ZipFileName,RetestDataFile,True)
										TempFlagRun=TempFlagRun or TempFlag
											
									if TempFlagRun:
										FlagRun=TempFlagRun
								###########################################
								#Print out the related customer data
								###########################################
								if not FlagRun:
									FlagRun=BHTC2CustData(LotPath,DeviceName,TestItemNum,TesterName,ProgramName,Version,LotWaferId,Patch,SortType,Operator,ChannelMap,StarTime,BinDefinePath,TempFileBHTC,TestItemNames)
									PassWaferLogFH=open(os.path.join(LogPath,"passwafer.log"),"a")
									PassWaferLogFH.write("%26s%5s%15s%5s%20s%5s%18s%5s%12s%5s" %(TimeConvert,"|",DeviceName,"|",Lot,"|",LotWaferId,"|",SortType,"\n"))
									PassWaferLogFH.close()
							else:
								ErrorFH=open(ErrorLogFile,"a")
								ErrorFH.write(TimeConvert),
								ErrorFH.write("\tNot found device:"),
								ErrorFH.write(DeviceName),
								ErrorFH.write("\'s configure data in file:"),
								ErrorFH.write(ConfigureFile),
								ErrorFH.write("! When Converting File:")
								ErrorFH.write(WaferFile)
								ErrorFH.write("\n")
								ErrorFH.close()
								#TempTime=time.asctime()
								#print TempTime
	
if __name__ == '__main__':       
	CurrentPath=os.getcwd()    
	ConfigPath=os.path.join(CurrentPath,"Config")
	ConfigFile=os.path.join(ConfigPath,"InitSetting_Auto") 
	CurrentTime=time.asctime()
	TempArray=re.split("\s+|:",CurrentTime)
	Hour=int(TempArray[3])
	Minute=int(TempArray[4])
	WaitTime=0  
	RunType=0
	RunInterval=2
	InitSettingFileFH=open(ConfigFile,'r')
	for Line in InitSettingFileFH.readlines():
		Line=Line.strip()
		if re.search("^#",Line,re.I):
			continue
		elif re.search("RunType=",Line,re.I):
			TempArray=re.split("=",Line)
			RunType=int(TempArray[1])
		elif re.search("RunInterval=",Line,re.I):
			TempArray=re.split("=",Line)
			RunInterval=float(TempArray[1])
	
	(MyInt,Residue)=divmod(Hour,2)                      
	if RunType:
		if Residue:
			WaitTime=1*60+(60-Minute)
		else:
			WaitTime=60-Minute
	else:
		if Residue:
			WaitTime=60-Minute
		else:
			WaitTime=1*60+(60-Minute)
	time.sleep(WaitTime*60)
	while True:                                                                         
		main()
		time.sleep(RunInterval*3600)                       